import multiprocessing
import sys
import time

import mysql.connector
import threading
from PyQt5.QtWidgets import QWidget, QVBoxLayout, QPushButton, QApplication,QHBoxLayout,QLineEdit,QTextEdit,QDesktopWidget
from PyQt5.QtGui import QTextCursor
from PyQt5.QtCore import pyqtSignal, QObject
import pymysql
from contextlib import contextmanager
import mysql.connector
from mysql.connector import Error as MySQLError
print(mysql.connector.__version__)
import queue
from app.interface_walmart.MySQLHelper_walmart import MySQLHelper
from followth_node import get_disk_serial_simple,getnodeorder,getnodescount
import threading
from datetime import datetime, timedelta, time as datetime_time
import requests
from lxml import etree
import time
import os
import random
from random import randint
from WalmartScraper import WalmartScraper
import re
import sys
import mysql.connector
from PyQt5.QtCore import QPoint
from PyQt5.QtWidgets import QWidget, QApplication
from PyQt5.QtCore import Qt  # 正确导入Qt
from datetime import datetime, timedelta, time as datetime_time
from PyQt5.QtWidgets import QApplication, QWidget, QVBoxLayout, QPushButton
import sys
from PyQt5.QtWidgets import QApplication, QWidget, QVBoxLayout, QTextEdit, QPushButton
from PyQt5.QtCore import pyqtSignal, QObject
from mysql.connector import Error as MySQLError  # 正确的导入方式
from followth_node import heartnode,getnodeorder,getnodescount   ,get_disk_serial_simple 
from PyQt5.QtCore import pyqtSignal, QObject
from PyQt5.QtWidgets import QApplication, QWidget, QVBoxLayout, QTextEdit, QPushButton
import traceback
from queue import Queue
from followth_node import get_disk_serial_simple,getnodeorder,getnodescount
import requests
from lxml import etree
import time
import os
import random
from random import randint
import threading
import mysql.connector
from datetime import datetime
from PyQt5.QtCore import pyqtSignal, QObject
from app.interface_walmart.MySQLHelper_walmart import MySQLHelper
import logging
import traceback
import MySQLdb
from MySQLdb.cursors import DictCursor
logging.basicConfig(filename=r'.\error.log', level=logging.ERROR)
error_details=''
splitchar = '->'
import sys
import os
import django
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'walmart_project.settings')
django.setup()
from app.models import Product
from walmartupc import get_upc
class MessageSender(QObject):
    textUpdateSignal = pyqtSignal(str)
 
class Window(QWidget):
    def __init__(self):
        super().__init__()

        layout_h = QHBoxLayout()
        self.dbhelper = MySQLHelper()
        self.resume_button = QPushButton("启动任务")
        self.resume_button.clicked.connect(self.resume_thread)
        self.pause_button = QPushButton("停止任务")
        self.pause_button.clicked.connect(self.pause_thread)
        self._pause_event = threading.Event()
        self._pause_event.clear()  #
        self.setGeometry(300, 300, 800, 500)
        self.setWindowTitle('关键词下载产品-RPA')
        text_layout = QHBoxLayout()
        self.textmemo = QTextEdit()
        text_layout.addWidget(self.textmemo)

        self.sender = MessageSender()
        self.queue = Queue()
        self.sender.textUpdateSignal.connect(self.textmemo.append)
        layout_h.addWidget(self.resume_button)

        self.WalmartScraper=WalmartScraper()
        self.queuelock=threading.Lock()
        tis2 = threading.Thread(target=self.process_queue)
        tis2.start()
        # self.textUpdateSignal.connect(self.textmemo.append)
        self.textmemo.append(
            '启动就可以下载关键词的对应数据')
        # layout_h.addWidget(self.pause_button)
        main_layout = QVBoxLayout()
        main_layout.addLayout(layout_h)
        main_layout.addLayout(text_layout)
        self.setLayout(main_layout)
        self.thread = []
        
        diskid=get_disk_serial_simple()
        tis = threading.Thread(target=self.downloaddata,args=(diskid,))
        self.thread.append(tis)
        
        self.resume_thread()

    def pause_thread(self):
        # self._pause_event.clear()
        ixx = 2

    def resume_thread(self):
        for i in range(1):
            self.thread[i].start()
        self.resume_button.setEnabled(False)
        # self._pause_event.set()
    def getexcept   (self,emsg):
        spec = [
            "'"
            '<', '>', '$', '*', '&', '%', '.', '"', '+', '-', '@', '"', '\\',
            '(', ')', ' ', '#', '.', '//', '#'
        ]
        for sp in spec:
            if sp in emsg:
                emsg = emsg.replace(sp, ' ')
        emsg = emsg.replace('"', '')
        return emsg.strip()
    def except_to_db(self,dbhelper,error_details):
        try:
            logmessage=f'{error_details}  {datetime.now()}'
            logmessage=self.getexcept(logmessage)
            # logging.error("An error occurred->" + logmessage, exc_info=True)   
            self.sender.textUpdateSignal.emit(logmessage)
            sql = f" insert into crawlexcept(opdate,exceptcon)  values(now(),%s)"
            dbhelper.execute(sql,(logmessage,)) 
        except Exception as e:
            print(f"处理异常出错: {str(e)}")
            
        
    def downloaddata(self,diskid):
            lock = threading.Lock()
            inat = 1
            try:
                while True:
                    self.sender.textUpdateSignal.emit(f'爬虫程序5分钟执行一次')
                    print('爬虫程序5分钟执行一次')
                    print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
                    lock.acquire()
                    try:
                        self.process_1(diskid)
                    except Exception as e:
                        traceback.print_exc()
                        error_details = traceback.format_exc()
                        self.except_to_db(self.dbhelper, error_details)                   
                    finally:
                        lock.release()
                    time.sleep(120)
            except Exception as e:
                print(e)
                error_details = traceback.format_exc()
                self.except_to_db(self.dbhelper, error_details)
                inat = inat + 1

    def getexcept   (self,emsg):
        spec = [
            "'"
            '<', '>', '$', '*', '&', '%', '.', '"', '+', '-', '@', '"', '\\',
            '(', ')', ' ', '#', '.', '//', '#'
        ]
        for sp in spec:
            if sp in emsg:
                emsg = emsg.replace(sp, ' ')
        emsg = emsg.replace('"', ' ')
        return emsg.strip()

    def getuser_bykeywords(self, keywords):
        try:
            query = f"SELECT DISTINCT username FROM app_keyword WHERE keyword = '{keywords}'  "
            results = self.dbhelper.query(query)
            usernames = [row['username'] for row in results if row['username']]  # 过滤掉空值
            return usernames
        except Exception as e:
            print(f"Error in getuser_bykeywords: {str(e)}")
            error_details = traceback.format_exc()
            self.except_to_db(self.dbhelper, error_details)
            return []
    def get_configure( self):
        result = self.dbhelper.query("SELECT * FROM configure")
        return result
    def process_1(self,diskid):
        inthoursdiff=24
        waitseconds=5
        current_time_stamp = time.time()
        local_time_struct = time.localtime(current_time_stamp)
        hour = local_time_struct.tm_hour
        minute = local_time_struct.tm_min
        intnewkeywords=1
        second = local_time_struct.tm_sec
        hrandom = random.randint(1, 30)
        print(f"当前时间是：{hour}时{minute}分{second}秒")
        current_date = datetime.now().date()
        target_date = current_date + timedelta(days=0)
        t8=datetime.combine(target_date, datetime_time(8, 0, 0))
        t11=datetime.combine(target_date, datetime_time(11, 0, 0))
        t20=datetime.combine(target_date, datetime_time(13, 0, 0))
        t23=datetime.combine(target_date, datetime_time(16, 0, 0))
        t1=None
        t2=None
        if hour <=11 and hour >=8:
                t1=t8
                t2=t11
        elif hour <=23 and hour >=17:
                t1=t20
                t2=t23     
        current_time = datetime.now()
        print(current_time) 
        configure=self.get_configure()
        intnewkeywordssleep=1
        intoldkeywrodssleep=29
        for row in configure:
            print(row)
            if row['cfgcode']=='newkeywordssleep':
                intnewkeywordssleep=int(row['cfgvalue'])
            if row['cfgcode']=='oldkeywordssleep':
                intoldkeywrodssleep=int(row['cfgvalue'])
            if row['cfgcode']=='hoursdiff':
                inthoursdiff=int(row['cfgvalue'])
                
        heartnode(self.dbhelper,diskid)
        nodeorder=getnodeorder(self.dbhelper,diskid)
        print('节点顺序',nodeorder)
        count=getnodescount(self.dbhelper)   
        print('节点数量',count)
        query = f"""SELECT distinct keyword  FROM app_keyword where IFNULL(is_prohibited,0)=0 and  lastdownloadtime is null     """
        if count>1:
            query=query + f" and  mod(id,{count} )={nodeorder}"
        query+=' limit 10'
        print(query)

        # 获取所有结果
        results_a = self.dbhelper.query(query)
        print(results_a)
        if len(results_a) ==0:
            intnewkeywords=0

            query = f"""SELECT distinct keyword  FROM app_keyword where IFNULL(is_prohibited,0)=0 and  TIMESTAMPDIFF(MINUTE, lastdownloadtime, NOW()) >= {inthoursdiff}*60    """

            if count>1:
                query=query + f" and  mod(sk.id,{count} )={nodeorder}"
            query+=' limit 10'
            print(query)
          
            intsleeptime=30
            results_a = self.dbhelper.query(query)
        data_to_insert = []

        links = []
        exceptitems = []
        link_withkey = []
         
        for index,results in enumerate( results_a):
            key2 = results['keyword']
            
            current_time_stamp = time.time()
            # 将时间戳转换为本地时间的结构体
            local_time_struct = time.localtime(current_time_stamp)
            # 提取小时、分钟、秒
            hour = local_time_struct.tm_hour
            minute = local_time_struct.tm_min
            second = local_time_struct.tm_sec
            hrandom = random.randint(1, 30)
            print(f"当前：{hour}时{minute}分{second}秒->",f'{index}/{len(results_a)} 个关键词记录{key2}')
            self.sender.textUpdateSignal.emit(f'当前：{hour}时{minute}分{second}秒-> {index+1}/{len(results_a)} 个关键词记录{key2}')
            # print(f'{index}/{len(results_a)} 个关键词记录{key2}')
            a = 1
            b = 0
            try:
                link1=self.WalmartScraper.search_products(key2, 6)
                usernames=self.getuser_bykeywords(key2)
                linkkey={
                    "key": key2,
                  
                    "links": link1,
                    'username':usernames
                }
                if len(link1)>0:
                    self.queue.put(linkkey)
                                        # self.process_2( linkkey)

            except Exception as e:
                error_details = traceback.format_exc()
                self.except_to_db(self.dbhelper, error_details)
                print(f"抓取walmart出错：{e}")
            time.sleep(waitseconds)
        if intnewkeywords==1:
            time.sleep(intnewkeywordssleep)
        else:
            time.sleep(intoldkeywrodssleep)#
        
            
        return link_withkey
    def process_queue(self):
        retry_count = 0
        max_retries = 3  # 最大重试次数
        
        while True:
            try:
                # 从队列中获取数据
                data=None
                with self.queuelock:
                    data = self.queue.get(timeout=300)  # 设置超时时间为 2 秒
                if data:
                    link = data
                    print(f"从队列中读取到：link={link},  ")
                    self.process_2(link)
            except Exception as e:
                print("队列已空，退出读取进程")
                traceback.print_exc()
                break
    def process_2(self,  link):
        current_time_stamp = time.time()
        local_time_struct = time.localtime(current_time_stamp)
        hour = local_time_struct.tm_hour
        minute = local_time_struct.tm_min
        second = local_time_struct.tm_sec
        second = local_time_struct.tm_sec
        hrandom = random.randint(1, 30)
        keywords = link['key']
        usernames = link['username']
        linksinner = link['links']
        keywordshis = keywords
        for kindex, lkx in enumerate(linksinner):
            try:
                product_info = self.WalmartScraper.get_walmart_upc(lkx['link'])
                upc = product_info['upc']
                detail_price = product_info['price']  # 获取详细页价格
                
                productid = lkx['SKU']
                brand = lkx['brand']
                title =self.getexcept( lkx['title'])
                title=title.replace('"', ' ').replace("'", ' ')
                itemlink = 'walmart.ca' + lkx['link']
                image = lkx['image']
                price = lkx['current_price'].replace('$', '').replace('current price Now ','')
                rating = lkx['rating']
                reviews = lkx['reviews']
                for username in usernames:
                    sql = f"select id,product_name,image,price,productid,rating,reviews,upc from app_product where productid = '{productid}' and username='{username}' "
                    results_b = self.dbhelper.query(sql)
                    intchanged = 0
                    if results_b:
                        print(keywords, '->序号：', kindex, '已经有数据了productid:', productid, '->username:', username)
                        title2 = results_b[0]['product_name']
                        pic2 = results_b[0]['image']
                        price2 = results_b[0]['price']
                        rating2 = results_b[0]['rating']
                        reviews2 = results_b[0]['reviews']
                        upc2 = results_b[0]['upc']
                        intchanged = 0
                        sql = f"update  app_product set  update_date=now() "
                        strq = ','
                        if pic2 != image:
                            sql += f" ,image='{image}' "
                            intchanged = 1
                        if title != title2:
                            sql += f" ,product_name='{title}' '"
                            intchanged = 1
                        current_price = detail_price or price  
                        diff = abs(float(price2) - float(current_price))
                        
                        if diff > 0.01:
                            sql += f' ,price={current_price} '
                            intchanged = 1
                        if rating != rating2:
                            sql += f' ,rating={rating} '
                            intchanged = 1
                        if reviews != reviews2:
                            sql += f' ,reviews={reviews} '
                            intchanged = 1
                        
                        if intchanged == 1:
                            sql += f' ,hasexcept="有异常"'
                        sql += f"  where productid='{productid}'"
                        if intchanged == 1:
                            print(sql)
                            self.dbhelper.updatesql(sql)
                    else:
                        print(keywords, '->序号：', kindex, '没有数据，需要插入。productid:', productid, '->username:', username)
                        insert_price = detail_price or price
                        sql = f"""
                            insert into app_product(is_following,username,productid,sku,keywords,product_name,link,image,price,rating,reviews,upc,create_date)
                            values(0,'{username}','{productid}','{productid}','{keywords}','{title}','{itemlink}','{image}',{insert_price},{rating},{reviews},'{upc}',now())
                        """
                        print(sql)
                        self.dbhelper.updatesql(sql)
            except Exception as e:
                print('process_2->出现异常', e)
                error_details = traceback.format_exc()
                self.except_to_db(self.dbhelper, error_details)
        for username in usernames:
            sql = f"""insert into sku_keywords_history(username,keywords,opdate) 
                        select username,keyword,now() from app_keyword where keyword = '{keywords}' and username='{username}'"""
            self.dbhelper.updatesql(sql)
        current_time_stamp = time.time()
        local_time_struct = time.localtime(current_time_stamp)
        hour = local_time_struct.tm_hour
        minute = local_time_struct.tm_min
        second = local_time_struct.tm_sec
        hrandom = random.randint(1, 30)
        print(keywords, '(关键词)->（店铺）', f"当前时间是：{hour}时{minute}分{second}秒")
         
if __name__ == "__main__":
    app = QApplication(sys.argv)
    win = Window()
    win.show()
    sys.exit(app.exec_())
